#!/bin/bash
# shellcheck disable=all # Jinja2 template can not be parsed with shellcheck

set -u -o pipefail

VERSION=0.4
CWD="$(realpath "${0%}")"
CWD="${CWD%/*}"

DAOS_INSTALL_DIR="{{ daos_runtime_dir }}/install"
DAOS_VENV_DIR="{{ daos_runtime_dir }}/virtualenvs"
DAOS_SOURCE_DIR="{{ daos_source_dir }}"
SERVERS_LIST="{{ groups["daos_servers"] | join(',') }}"
{% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
CLIENTS_LIST="{{ groups["daos_clients"] | join(',') }}"
{% endif %}

SCONS_EXE=scons
SSH_EXE=ssh
CLUSH_EXE=clush
NODESET_EXE=nodeset
MODULE_EXE=module
CMAKE_EXE=cmake3
MAKE_EXE=make
WGET_EXE=wget
PYTHON_EXE=python3.11
PIP_EXE=pip

CLUSH_OPTS="-bL -S"

MPI_FILE_UTILS_DEPS=("https://github.com/hpc/libcircle/releases/download/v0.3/libcircle-0.3.0.tar.gz"
	"https://github.com/llnl/lwgrp/releases/download/v1.0.4/lwgrp-1.0.4.tar.gz"
	"https://github.com/llnl/dtcmp/releases/download/v1.1.4/dtcmp-1.1.4.tar.gz"
	"https://github.com/libarchive/libarchive/releases/download/v3.5.1/libarchive-3.5.1.tar.gz")

ANSI_COLOR_BLACK=30
ANSI_COLOR_RED=31
ANSI_COLOR_GREEN=32
ANSI_COLOR_YELLOW=33
ANSI_COLOR_BLUE=34
ANSI_COLOR_MAGENTA=35
ANSI_COLOR_CYAN=36
ANSI_COLOR_WHITE=37
ANSI_COLOR_BRIGHT_BLACK=90
ANSI_COLOR_BRIGHT_RED=91
ANSI_COLOR_BRIGHT_GREEN=92
ANSI_COLOR_BRIGHT_YELLOW=93
ANSI_COLOR_BRIGHT_BLUE=94
ANSI_COLOR_BRIGHT_MAGENTA=95
ANSI_COLOR_BRIGHT_CYAN=96
ANSI_COLOR_BRIGHT_WHITE=97

TRACE_LEVEL_QUIET=-1
TRACE_LEVEL_STANDARD=0
TRACE_LEVEL_VERBOSE=1
TRACE_LEVEL_DEBUG=2
TRACE_LEVEL=$TRACE_LEVEL_STANDARD

DAOS_BUILD_DEPS=false
DAOS_BUILD_TYPE=debug
JOBS_NB=1
FORCE_INSTALL=false

function debug
{
	if [[ $TRACE_LEVEL -ge $TRACE_LEVEL_DEBUG ]]
	then
		echo -e "[\e[${ANSI_COLOR_GREEN}mDEBUG  \e[00m] $@"
	fi
}


function info
{
	if [[ $TRACE_LEVEL -ge $TRACE_LEVEL_VERBOSE ]]
	then
		echo -e "[\e[${ANSI_COLOR_CYAN}mINFO   \e[00m] $@"
	fi
}

function warning
{
	if [[ $TRACE_LEVEL -ge $TRACE_LEVEL_STANDARD ]]
	then
		echo -e "[\e[${ANSI_COLOR_YELLOW}mWARNING\e[00m] $@" 1>&2
	fi
}

function error
{
	if [[ $TRACE_LEVEL -ge $TRACE_LEVEL_STANDARD ]]
	then
		echo -e "[\e[${ANSI_COLOR_BRIGHT_RED}mERROR  \e[00m] $@" 1>&2
	fi
}

function fatal
{
	if [[ $TRACE_LEVEL -ge $TRACE_LEVEL_STANDARD ]]
	then
		echo -e "[\e[${ANSI_COLOR_RED}mFATAL  \e[00m] $@" 1>&2
	fi
	exit 1
}

function check_cmds
{
	hash -r
	for cmd in $@
	do
		{ hash $cmd > "/dev/null" 2>&1 ; } || { fatal "$cmd command not installed" ; }
	done
}

function usage
{
	cat <<- EOF
		usage: daos-make.sh [OPTIONS]

		Build and install DAOS for running avocado functional tests

		Options:
		       --mpich <path>           Path of mpich source tree
		       --hdf5 <path>		Path of hdf5 source tree
		       --vol-daos <path>	Path of vol-daos source tree
		       --mpifileutils <path>    Path of mpi file utils source tree
		       --ior <path>             Path of ior file utils source tree
		       --fio <path>             Path of fio file utils source tree
		   -b, --build-type <type>      DAOS build type: debug, release or dev
		   -d, --deps                   Build DAOS dependencies
		   -j, --jobs <job>             Allow N jobs at once; one job with no arg
		   -f, --force                  Overwrite the current install of DAOS and deps
		   -h, --help                   Show this help message and exit
		   -V, --version                Show version number
		   -q, --quiet                  Quiet mode
		   -v, --verbose                Verbose mode
		   -D, --debug                  Debug mode
	EOF
}

function run
{
	if [[ $TRACE_LEVEL -ge $TRACE_LEVEL_STANDARD ]]
	then
		"$@"
	else
		"$@" &> /dev/null
	fi
}

OPTIONS=$(getopt -o "b:dj:fhVvDq" --long "mpich:,hdf5:,vol-daos:,mpifileutils:,ior:,fio:,build-type:,deps,jobs:,force,help,version,verbose,debug,quiet" -- "$@") || exit 1
eval set -- "$OPTIONS"
while true
do
	case "$1" in
		--mpich) MPICH_PATH="$2" ; shift 2 ;;
		--hdf5) HDF5_PATH="$2" ; shift 2 ;;
		--vol-daos) VOL_DAOS_PATH="$2" ; shift 2 ;;
		--mpifileutils) MPI_FILE_UTILS_PATH="$2" ; shift 2 ;;
		--ior) IOR_PATH="$2" ; shift 2 ;;
		--fio) FIO_PATH="$2" ; shift 2 ;;
		-b|--build-type) DAOS_BUILD_TYPE="$2" ; shift 2 ;;
		-d|--deps) DAOS_BUILD_DEPS=true ; shift 1 ;;
		-j|--jobs) JOBS_NB="$2" ; shift 2 ;;
		-f|--force) FORCE_INSTALL=true ; shift 1;;
		-h|--help) usage ; exit 0;;
		-V|--version) echo "daos-make.sh version=$VERSION" ; exit 0 ;;
		-v|--verbose) TRACE_LEVEL=$TRACE_LEVEL_VERBOSE ; shift 1 ;;
		-D|--debug) TRACE_LEVEL=$TRACE_LEVEL_DEBUG ; set -x ; shift 1 ;;
		-q|--quiet) TRACE_LEVEL=$TRACE_LEVEL_QUIET ; shift 1 ;;
		--) shift ; break ;;
		*) fatal "unrecognized command line option: $1" ;;
	esac
done

check_cmds $SSH_EXE $WGET_EXE $CLUSH_EXE $NODESET_EXE $MODULE_EXE $MAKE_EXE $CMAKE_EXE

set -e

info "Checking of home partition"
{% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
for host in $($NODESET_EXE -e $CLIENTS_LIST $SERVERS_LIST)
{% else %}
for host in $($NODESET_EXE -e $SERVERS_LIST)
{% endif %}
do
	run $SSH_EXE root@$host mountpoint --quiet "$HOME" || fatal "Home directory '$home' not mounted on $host"
done

if "$FORCE_INSTALL" ; then
	for dir in "$DAOS_INSTALL_DIR" "$DAOS_VENV_DIR" "$DAOS_SOURCE_DIR/build"
	do
		info "Removing old DAOS directory $dir"
		if ! run /bin/rm -fr "$dir" ; then
			fatal "Old install directory $dir could not be removed"
		fi
	done
fi

if [[ ! -d "$DAOS_INSTALL_DIR" ]] ; then
	info "Creating DAOS install directory $DAOS_INSTALL_DIR"
	if ! run mkdir -p "$DAOS_INSTALL_DIR" ; then
		fatal "DAOS install directory $DAOS_INSTALL_DIR could not be created"
	fi
fi

if [[ ! -d "$DAOS_VENV_DIR" ]] ; then
	info "Creating DAOS python virtualenv directory $DAOS_VENV_DIR"
	if ! run mkdir -p "$DAOS_VENV_DIR" ; then
		fatal "DAOS python virtualenv directory $DAOS_VENV_DIR could not be created"
	fi
fi

if [[ ! -f "$DAOS_VENV_DIR/bin/activate" ]] ; then
	info "Creating DAOS python virtualenv $DAOS_VENV_DIR"
	run $PYTHON_EXE -m venv "$DAOS_VENV_DIR"
fi
info "Installing DAOS python virtualenv $DAOS_VENV_DIR"
source "$DAOS_VENV_DIR/bin/activate"
run $PIP_EXE install --upgrade pip
run $PIP_EXE install -r "$DAOS_SOURCE_DIR/requirements.txt"

SCONS_OPTS="--directory="$DAOS_SOURCE_DIR" --jobs=$JOBS_NB"

if "$DAOS_BUILD_DEPS" ; then
	info "Building DAOS dependencies from source tree $DAOS_SOURCE_DIR"
	if ! run "$SCONS_EXE" BUILD_TYPE="$DAOS_BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" $SCONS_OPTS --build-deps=only ; then
		fatal "DAOS dependencies could not be properly build"
	fi
fi

info "Building DAOS from source tree $DAOS_SOURCE_DIR"
# NOTE Dependencies will not be build as 'no' is default value of the --build-deps option
if ! run env MPI_PKG=any "$SCONS_EXE" BUILD_TYPE="$DAOS_BUILD_TYPE" PREFIX="$DAOS_INSTALL_DIR" $SCONS_OPTS ; then
	fatal "DAOS could not be properly build"
fi

info "Installing DAOS to $DAOS_INSTALL_DIR"
source "$DAOS_VENV_DIR/bin/activate"
if ! run "$SCONS_EXE" install $SCONS_OPTS ; then
	fatal "DAOS could not be properly installed"
fi

deactivate

info "Finalizing install of ftest setup scripts"
for file_path in "$DAOS_SOURCE_DIR"/.build_vars*
do
	file_name=$(basename $file_path)
	rm -fv "$DAOS_INSTALL_DIR/lib/daos/$file_name"
	run cp -av "$file_path" "$DAOS_INSTALL_DIR/lib/daos/$file_name"
done

info "Finalizing spdk install"
for item in $(find "$DAOS_INSTALL_DIR/share/daos" -type f)
do
        dest="/usr/${item#$DAOS_INSTALL_DIR/}"
        run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST install -v -D -o root -g root -m 755 "$item" "$dest"
done

info "Finalizing daos-server-helper install"
run chmod 0446 "$DAOS_INSTALL_DIR/bin/daos_server_helper"
run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST cp -f "$DAOS_INSTALL_DIR/bin/daos_server_helper" /usr/bin
run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST chown root:daos_server /usr/bin/daos_server_helper
run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST chmod 4755 /usr/bin/daos_server_helper
run $CLUSH_EXE $CLUSH_OPTS -w $SERVERS_LIST chmod a-x  "$DAOS_INSTALL_DIR/bin/daos_server_helper"

{% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
info "Finalizing dfuse install"
run $CLUSH_EXE $CLUSH_OPTS -l root -w $CLIENTS_LIST cp -f "$DAOS_INSTALL_DIR/bin/dfuse" /usr/bin
run $CLUSH_EXE $CLUSH_OPTS -l root -w $CLIENTS_LIST chown root:root /usr/bin/dfuse
run $CLUSH_EXE $CLUSH_OPTS -l root -w $CLIENTS_LIST chmod 755 /usr/bin/dfuse
{% endif %}

info "Updating dynamic linker configuration"
{% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST -w $CLIENTS_LIST rm -f /etc/ld.so.cache
run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST -w $CLIENTS_LIST ldconfig
{% else %}
run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST rm -f /etc/ld.so.cache
run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST ldconfig
{% endif %}

if [[ ${MPICH_PATH:+x} ]] ; then
	info "Building and installing MPICH from source tree $MPICH_PATH to $DAOS_INSTALL_DIR"

	if [[ ! -f "$MPICH_PATH/mpich-doxygen.in" ]] ; then
		fatal "Invalid MPICH directory: path=$MPICH_PATH"
	fi

	if "$FORCE_INSTALL" ; then
		info "Removing old MPICH install"
		if ! run rm -fr "$MPICH_PATH/build" "$MPICH_PATH/configure" ; then
			fatal "Old install of MPICH could not be removed"
		fi
	fi

	if [[ ! -f "$MPICH_PATH/configure" ]] ; then
		pushd "$MPICH_PATH" > /dev/null
		run env MPI_LIB="" ./autogen.sh
		popd > /dev/null
	fi

	run mkdir -p "$MPICH_PATH/build"
	pushd "$MPICH_PATH/build"  > /dev/null

	if [[ ! -f "$MPICH_PATH/build/Makefile" ]] ; then
		MPICH_CONFIGURE_OPTS=( --prefix="$DAOS_INSTALL_DIR"
			--enable-g=all
			--enable-debuginfo
			--enable-sharedlibs=gcc
			--enable-shared
			--enable-static=no
			--enable-lib-depend
			--disable-rpath
			--disable-silent-rules
			--enable-fc
			--with-device=ch3:nemesis
			--with-hwloc=embedded
			--enable-fortran=all
			--enable-romio
			--with-file-system=ufs+daos
			--with-daos="$DAOS_INSTALL_DIR"
			--disable-checkerrors
			--disable-perftest
			--disable-large-tests
			--disable-ft-tests
			--disable-comm-overlap-tests
			--enable-threads=single )

		run env MPI_LIB="" ../configure "${MPICH_CONFIGURE_OPTS[@]}"
	fi

	run env MPI_LIB="" $MAKE_EXE -j $JOBS_NB install
	popd > /dev/null

	run rm -f "$DAOS_INSTALL_DIR/share/modulefiles/mpich-x86_64"
	run mkdir -p "$DAOS_INSTALL_DIR/share/modulefiles"
	cat > "$DAOS_INSTALL_DIR/share/modulefiles/mpich-x86_64" <<- EOF
	#%Module 1.0
	#
	#  MPICH module for use with 'environment-modules' package:
	#

	# Only allow one mpi module to be loaded at a time
	conflict mpi

	# Define prefix so PATH and MANPATH can be updated.
	setenv        MPI_BIN       $DAOS_INSTALL_DIR/bin
	setenv        MPI_SYSCONFIG $DAOS_INSTALL_DIR/etc
	setenv        MPI_FORTRAN_MOD_DIR $DAOS_INSTALL_DIR/include
	setenv        MPI_INCLUDE   $DAOS_INSTALL_DIR/include
	setenv        MPI_LIB       $DAOS_INSTALL_DIR/lib
	setenv        MPI_MAN       $DAOS_INSTALL_DIR/man
	setenv        MPI_PYTHON_SITEARCH       @py2sitearch@/mpich
	setenv        MPI_PYTHON2_SITEARCH      @py2sitearch@/mpich
	setenv        MPI_PYTHON3_SITEARCH      /usr/lib64/python3.6/site-packages/mpich
	setenv        MPI_COMPILER  mpich-x86_64
	setenv        MPI_SUFFIX    _mpich
	setenv        MPI_HOME      $DAOS_INSTALL_DIR
	prepend-path  PATH          $DAOS_INSTALL_DIR/bin
	prepend-path  LD_LIBRARY_PATH $DAOS_INSTALL_DIR/lib
	prepend-path  MANPATH       :$DAOS_INSTALL_DIR/man
	prepend-path  PKG_CONFIG_PATH $DAOS_INSTALL_DIR/man/lib/pkgconfig
	EOF

	{
		cat <<- EOF
		set -e -o pipefail

		if [[ -f /usr/share/modulefiles/mpi/mpich-x86_64 && ! -f /usr/share/modulefiles/mpi/mpich-x86_64.orig ]] ; then
			echo "[WARNING] MPICH already installed: backup module file /usr/share/modulefiles/mpi/mpich-x86_64.orig"
			mv /usr/share/modulefiles/mpi/mpich-x86_64 /usr/share/modulefiles/mpi/mpich-x86_64.orig
		fi
		mkdir -p /usr/share/modulefiles/mpi
		ln -fs "$DAOS_INSTALL_DIR/share/modulefiles/mpich-x86_64" /usr/share/modulefiles/mpi/mpich-x86_64
		EOF
	} | run $CLUSH_EXE $CLUSH_OPTS -l root -w $CLIENTS_LIST -w $SERVERS_LIST bash -s
fi

if [[ ${HDF5_PATH:+x} ]] ; then
	info "Building and installing HDF5 from source tree $HDF5_PATH to $DAOS_INSTALL_DIR"

	if [[ ! -f "$HDF5_PATH/config/libhdf5.pc.in" ]] ; then
		fatal "Invalid HDF5 directory: path=$HDF5_PATH"
	fi

	if "$FORCE_INSTALL" ; then
		info "Removing old HDF5 install"
		if ! run rm -fr "$HDF5_PATH/build" "$HDF5_PATH/configure" ; then
			fatal "Old install of HDF5 could not be removed"
		fi
	fi

	run $MODULE_EXE load mpi/mpich-x86_64

	if [[ ! -f "$HDF5_PATH/configure" ]] ; then
		pushd "$HDF5_PATH" > /dev/null
		run ./autogen.sh
		popd > /dev/null
	fi

	run mkdir -p "$HDF5_PATH/build"
	pushd "$HDF5_PATH/build" > /dev/null

	if [[ ! -f "$HDF5_PATH/build/Makefile" ]] ; then
		HDF5_CONFIGURE_OPTS=( --prefix="$DAOS_INSTALL_DIR"
			--disable-dependency-tracking
			--disable-silent-rules
			--enable-build-mode=debug
			--enable-fortran
			--enable-hl
			--enable-shared
			--enable-parallel
			--enable-map-api )

		run ../configure "${HDF5_CONFIGURE_OPTS[@]}"
	fi

	run $MAKE_EXE -j $JOBS_NB install
	popd > /dev/null
fi

if [[ ${VOL_DAOS_PATH:+x} ]] ; then
	info "Building and installing DAOS HDF5 Volume from source tree $VOL_DAOS_PATH to $DAOS_INSTALL_DIR"

	if [[ ! -f "$VOL_DAOS_PATH/CMake/hdf5_vol_daos.pc.in" ]] ; then
		fatal "Invalid DAOS HDF5 Volume directory: path=$VOL_DAOS_PATH"
	fi

	if "$FORCE_INSTALL" ; then
		info "Removing old DAOS HDF5 Volume install"
		if ! run rm -fr "$VOL_DAOS_PATH/build" ; then
			fatal "Old install of HDF5 could not be removed"
		fi
	fi

	run $MODULE_EXE load mpi/mpich-x86_64

	run mkdir -p "$VOL_DAOS_PATH/build"
	pushd "$VOL_DAOS_PATH/build" > /dev/null

	if [[ ! -f "$VOL_DAOS_PATH/build/Makefile" ]] ; then
		VOL_DAOS_CMAKE_OPTS=( -DDCMAKE_BUILD_TYPE=Debug
			-DCMAKE_INSTALL_PREFIX="$DAOS_INSTALL_DIR"
			-DCMAKE_SKIP_RPATH:BOOL=ON
			-DDAOS_INCLUDE_DIR="$DAOS_INSTALL_DIR/include/"
			-DDAOS_LIBRARY="$DAOS_INSTALL_DIR/lib64/libdaos.so"
			-DDAOS_UNS_LIBRARY="$DAOS_INSTALL_DIR/lib64/libduns.so"
			-DBUILD_TESTING=ON
			-DBUILD_EXAMPLES=ON
			-DHDF5_VOL_TEST_ENABLE_PART=ON
			-DHDF5_VOL_TEST_ENABLE_PARALLEL=ON
			-DHDF5_VOL_TEST_ENABLE_ASYNC=ON
			-DHDF5_VOL_DAOS_USE_SYSTEM_HDF5=OFF
			-DMPI_C_COMPILER="$MPI_BIN/mpicc" )

			run $CMAKE_EXE "${VOL_DAOS_CMAKE_OPTS[@]}" ..
	fi

	run $MAKE_EXE -j $JOBS_NB install

	for file_path in bin/*test* ; do
		install --verbose --mode=0755 $file_path "$DAOS_INSTALL_DIR/bin"
	done

	popd > /dev/null
fi

if [[ ${MPI_FILE_UTILS_PATH:+x} ]] ; then
	info "Building and installing MPI File Utils from source tree $MPI_FILE_UTILS_PATH to $DAOS_INSTALL_DIR"

	if [[ ! -f "$MPI_FILE_UTILS_PATH/mpifileutils.spec" ]] ; then
		fatal "Invalid MPI File Utils directory: path=$MPI_FILE_UTILS_PATH"
	fi

	if "$FORCE_INSTALL" ; then
		info "Removing old MPI File Utils install"
		if ! run rm -fr "$MPI_FILE_UTILS_PATH/deps" "$MPI_FILE_UTILS_PATH/build" "$MPI_FILE_UTILS_PATH/configure" ; then
			fatal "Old install of MPI File Utils could not be removed"
		fi
	fi

	run $MODULE_EXE load mpi/mpich-x86_64

	run mkdir -p "$MPI_FILE_UTILS_PATH/deps" "$MPI_FILE_UTILS_PATH/build"

	for dep_uri in "${MPI_FILE_UTILS_DEPS[@]}" ; do
		pushd "$MPI_FILE_UTILS_PATH/deps" > /dev/null

		file_name=$(basename "$dep_uri")
		dep_name=$(basename "$dep_uri" .tar.gz)
		info "Building and installing MPI File Utils dependency $dep_name"

		if [[ ! -d $dep_name ]] ; then
			run $WGET_EXE "$dep_uri"
			run tar xzf "$file_name"
		fi

		cd $dep_name

		if [[ ! -f Makefile ]] ; then
			opts="--prefix=$DAOS_INSTALL_DIR"
			if [[ $dep_name =~ dtcmp ]] ; then
				opts="$opts --with-lwgrp=$DAOS_INSTALL_DIR"
			fi
			run env LDFLAGS="-L$DAOS_INSTALL_DIR/lib64 -L$DAOS_INSTALL_DIR/lib" ./configure $opts
		fi

		run env LDFLAGS="-L$DAOS_INSTALL_DIR/lib64 -L$DAOS_INSTALL_DIR/lib" $MAKE_EXE -j $JOBS_NB install

		popd > /dev/null
	done

	pushd "$MPI_FILE_UTILS_PATH/build" > /dev/null

	if [[ ! -f "$MPI_FILE_UTILS_PATH/build/Makefile" ]] ; then
		MPI_FILE_UTILS_CMAKE_OPTS=( -DCMAKE_BUILD_TYPE=Debug
			-DENABLE_DAOS=ON
			-DENABLE_HDF5=ON
			-DENABLE_LIBARCHIVE=ON
			-DCMAKE_INSTALL_PREFIX="$DAOS_INSTALL_DIR"
			-DWITH_DAOS_PREFIX="$DAOS_INSTALL_DIR"
			-DWITH_DTCMP_PREFIX="$DAOS_INSTALL_DIR"
			-DWITH_LibCircle_PREFIX="$DAOS_INSTALL_DIR"
			-DWITH_LibArchive_PREFIX="$DAOS_INSTALL_DIR" )
		if [[ -h "$DAOS_INSTALL_DIR/lib/libhdf5.so" ]] ; then
			MPI_FILE_UTILS_CMAKE_OPTS+=( -DWITH_HDF5_PREFIX="$DAOS_INSTALL_DIR" )
		else
			MPI_FILE_UTILS_CMAKE_OPTS+=( -DWITH_HDF5_PREFIX=/usr )
		fi

		run $CMAKE_EXE "${MPI_FILE_UTILS_CMAKE_OPTS[@]}" ..
	fi

	run $MAKE_EXE -j $JOBS_NB install

	popd > /dev/null
fi

if [[ ${IOR_PATH:+x} ]] ; then
	info "Building and installing IOR from source tree $IOR_PATH to $DAOS_INSTALL_DIR"

	if [[ ! -f "$IOR_PATH/src/ior.c" ]] ; then
		fatal "Invalid IOR directory: path=$IOR_PATH"
	fi

	if "$FORCE_INSTALL" ; then
		info "Removing old IOR install"
		if ! run rm -fr "$IOR_PATH/build" "$IOR_PATH/configure" ; then
			fatal "Old install of IOR could not be removed"
		fi
	fi

	run $MODULE_EXE load mpi/mpich-x86_64

	if [[ ! -f "$IOR_PATH/configure" ]] ; then
		pushd "$IOR_PATH" > /dev/null
		run env MPI_LIB="" ./bootstrap
		popd > /dev/null
	fi

	run mkdir -p "$IOR_PATH/build"
	pushd "$IOR_PATH/build" > /dev/null

	if [[ ! -f "$IOR_PATH/build/Makefile" ]] ; then
		IOR_CONFIGURE_OPTS=( --prefix="$DAOS_INSTALL_DIR"
			--disable-dependency-tracking
			--with-daos="$DAOS_INSTALL_DIR"
			--with-hdf5
			--with-mpiio)

		run ../configure "${IOR_CONFIGURE_OPTS[@]}"
	fi

	run env CFLAGS="-g -O0" make -j $(nproc) install

	popd > /dev/null
fi

if [[ ${FIO_PATH:+x} ]] ; then
	info "Building and installing FIO from source tree $FIO_PATH to $DAOS_INSTALL_DIR"

	if [[ ! -f "$FIO_PATH/fio.c" ]] ; then
		fatal "Invalid FIO: path=$FIO_PATH"
	fi

	if "$FORCE_INSTALL" ; then
		info "Removing old FIO install"
		if ! run rm -fr "$FIO_PATH/build" ; then
			fatal "Old install of FIO could not be removed"
		fi
	fi

	run mkdir -p "$FIO_PATH/build"
	pushd "$FIO_PATH/build" > /dev/null

	if [[ ! -f "$FIO_PATH/build/Makefile" ]] ; then
		run env CFLAGS="-I$DAOS_INSTALL_DIR/include" LDFLAGS="-L$DAOS_INSTALL_DIR/lib64" ../configure --prefix="$DAOS_INSTALL_DIR"
	fi

	run env CFLAGS="-g -O0" make -j $(nproc) install

	popd > /dev/null

	pushd "$FIO_PATH/doc" > /dev/null

	FIO_VENV_DIR="$FIO_PATH/build/virtualenvs"
	if [[ ! -f "$FIO_VENV_DIR/bin/activate" ]] ; then
		info "Creating FIO python virtualenv $FIO_VENV_DIR"
		run $PYTHON_EXE -m venv "$FIO_VENV_DIR"
	fi

	info "Installing FIO python virtualenv $FIO_VENV_DIR"
	source "$FIO_VENV_DIR/bin/activate"
	run $PIP_EXE install --upgrade pip
	run $PIP_EXE install sphinx

	run make man
	run install -v output/man/fio.1 $DAOS_INSTALL_DIR/man/man1/fio.1

	deactivate

	popd
fi

info "Updating dynamic linker configuration"
{% if "daos_clients" in groups and groups["daos_clients"] | length > 0 %}
run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST -w $CLIENTS_LIST rm -f /etc/ld.so.cache
run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST -w $CLIENTS_LIST ldconfig
{% else %}
run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST rm -f /etc/ld.so.cache
run $CLUSH_EXE $CLUSH_OPTS -l root -w $SERVERS_LIST ldconfig
{% endif %}
